# coding:utf-8

# 需要pip安装selenium
from selenium import webdriver
from selenium.webdriver.common.by import By
import time
import json


# 处理得到的post信息，并存储在字典中
def splitPost(information):
    redata = {}
    status = True
    strdata = information.split('\n')
    post = {}
    post['voted'] = strdata.pop(0)
    try:
        post['author'] = strdata.pop(0).split(' ')[1][2::]
    except IndexError:
        pass
    post['send_time'] = strdata.pop(0)
    post['title'] = strdata.pop(0)
    try:
        strdata.pop(-1)
        strdata.pop(-1)
    except:
        status = False
        print(information)
    try:
        post['comment_num'] = strdata.pop(-1).split(' ')[0]
    except:
        status = False
        post['comment_num'] = 0
        print(information)
    article = ""
    for s in strdata:
        article = article + s
    post['article'] = article
    #post['url'] = href

    redata['status'] = status
    if status:
        redata['data'] = post
    else:
        redata['data'] = ""
    return redata


# 重试，等待网页相应进行处理
def failRetry(driver, faildata_list, fail_but_ok, allpost):
    posts = driver.find_elements(By.XPATH, '//div[@data-testid="post-container"]')
    #hrefs = driver.find_elements(By.XPATH, '//a[@class="_3jOxDPIQ0KaOWpzvSQo-1s" and @data-click-id="timestamp"]')
    if len(posts[i].text) == 0:
        print("仍未加载，记录编号！")
        faildata_list.append(i)
    else:
        fail_but_ok += 1
        redata = splitPost(posts[i].text)
        if redata['status']:
            allpost.append(post)
            print(post)
            print("获取成功")
        else:
            print("信息仍然处理失败")
    pass


# 设置脚本开启时间
starttime = time.time()
startform = time.asctime(time.localtime(starttime))
print("脚本开启时间：", startform)

# 创建Firefox对象，上面是Firefox，下面是Chrome，看需求引用。需要插件配合
# firefox的插件地址：https://link.segmentfault.com/?enc=Bj7UXkIAK3vOWgcW77eNFg%3D%3D.FIHLSNQzM61tIt%2FTuF47p2DbLJQNfPlm9rT5E8eHbqJRpPkv5IiqZykoCwWVcRSo
# chrome的插件地址：https://link.segmentfault.com/?enc=cOztCMKAu0lOZ0JVlFrExg%3D%3D.PkL%2F4fUnKWfOf413ooOxGjMkX9tqzDVHhMva1LRy7Frtl0LtAC28IESd28HxTJiZ4r27TBixlXSAapjGUIEeqA%3D%3D
# 下载解压完，丢到python的目录里就行
# 文件夹中的geckodriver对应firefox，chromedriver对应chrome
#driver = webdriver.Chrome(executable_path="C:/crawler/chromedriver.exe")
driver = webdriver.Chrome()

# 获取网页，等网页加载完，才能进行下一步
driver.get('https://www.reddit.com/r/Cooking/')

# 根据xpath，去截取想要的
posts = driver.find_elements(By.XPATH, '//div[@data-testid="post-container"]')
print("当前数据条数：", len(posts))
# js语言的返回网页高度
js = "return action=document.body.scrollHeight"
# 执行上述js语句
height = driver.execute_script(js)

# 加载次数
load_num = 0
# 加载状态
status = True
# 重新尝试加载次数
retry_num = 0
# 加载失败的post的次数
fail_num = 0
# 加载失败的post的编号
faildata_list = []
# 后加载成功的post的次数
fail_but_ok = 0
# post处理失败次数
fail_split_num = 0

posts = driver.find_elements(By.XPATH, '//div[@data-testid="post-container"]')
data_length = len(posts)

print('当前页面数据条数：', data_length)
print('开始滚动加载页面。。。')
localtime = time.asctime(time.localtime(time.time()))
print("滚动条滚动时间：", localtime)

# 所有post的集合
allpost = []

# 滚动时，通过高度判断滚动是否结束
while status:
    # 将滚动条调整至页面底部
    driver.execute_script('window.scrollTo(0,1000000)')
    time.sleep(30)
    newheight = driver.execute_script(js)

    if newheight > height:
        # 重置尝试次数
        retry_num = 0
        load_num += 1
        height = newheight
        posts = driver.find_elements(By.XPATH, '//div[@data-testid="post-container"]')
        #hrefs = driver.find_elements(By.XPATH, '//a[@class="_3jOxDPIQ0KaOWpzvSQo-1s" and @data-click-id="timestamp"]')
        now_len = len(posts)
        for i in range(data_length - 1, now_len):
            print("------------------------------------------------")

            # 跳到每个post的位置
            post_height = posts[i].rect['y']
            post_js = 'window.scrollTo(0,' + str(post_height) + ')'
            driver.execute_script(post_js)
            print("第", i, "条post：")

            # 当获取信息为空时，等待5s，重新获取一边
            if len(posts[i].text) == 0:
                fail_num += 1
                print("还未加载，等待重新获取信息！")
                time.sleep(5)
                failRetry(driver, faildata_list, fail_but_ok, allpost)
            # 当信息不为空时
            else:
                print('###########################1111')
                print(posts[i].text)
                print('###########################222')
                redata = splitPost(posts[i].text)
                if redata['status']:
                    allpost.append(redata['data'])
                    print(redata['data'])
                    # print("获取成功")
                else:
                    fail_split_num += 1
                    print("第", i, "条信息处理失败")

            print("------------------------------------------------")
        data_length = now_len
        print("当前数据条数：", data_length)
        print("加载失败post数量：", fail_num)
        print("加载又成功post数量：", fail_but_ok)
        print("信息处理失败次数：", fail_split_num)

    else:
        if retry_num < 3:
            retry_num += 1
            print("重试第", retry_num, "次")
            time.sleep(40)
        else:
            status = False
            print("滚动加载页面结束！")
            print("加载次数:", load_num)
            print("数据总数：", data_length)
            print("加载失败数量：", fail_num)
            print("信息处理失败次数：", fail_split_num)

postjson = json.dumps(allpost, sort_keys=True, indent=4, separators=(',', ': '))
filename = "Cooking22.1.02666.json"
with open(filename, 'w') as f:
    f.write(postjson)

endtime = time.time()
endform = time.asctime(time.localtime(endtime))
print("脚本结束时间：", endform)
hastime = endtime - starttime
print("脚本总用时：", time.strftime("%M:%S", time.localtime(hastime)))

driver.quit()
